package com.chance.cc.crawler.development.scripts.sohu.carSeries;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;

public class SohuCarSeriesCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(SohuCarSeriesCrawlerScript.class);//日志
    private static final String DOMAIN = "sohu";
    private static final String SITE = "carSeries";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";

    //   https://car.autohome.com.cn/javascript/NewSpecCompare.js?20131010
    private static final String HomeRegulars = "https://car.autohome.com.cn/javascript/NewSpecCompare.js\\?20131010";//动态链接
    //    http://db.auto.sohu.com/porsche-1109/1959
    private static final String CarSeriesRegulars = "http://db.auto.sohu.com/\\S*/\\S*";//车系链接


    @Override
    public String domain() {
        return DOMAIN;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(HomeRegulars);
        addUrlRegular(CarSeriesRegulars);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type + ".msg").get();
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> keys = new ArrayList<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all) {
            JSONObject jsonObject = JSONObject.parseObject(data);
            String keyword = jsonObject.getString("keyword");
            keys.add(keyword);
        }
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("keys", keys);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();
        //        判断页面是否加载成功
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (page.getStatusCode() != 200 || (!page.isDownloadSuccess())) {
            log.error("download page url == {null} error status is {}", requestUrl, page.getStatusCode());
            if (page.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
                crawlerRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(HomeRegulars)) {
            homeRequest(crawlerRecord, page, parsedLinks);//解析动态车系
        } else
        if (crawlerRecord.getHttpRequest().getUrl().matches(CarSeriesRegulars)) {
            carSeriesRequest(crawlerRecord, page, parsedLinks);//解析详情和在售配置
        }
        return parsedLinks;
    }

    //    解析动态车系
    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
        List<String> name = (List<String>) keys.getVal();
        for (int i = 0; i < name.size(); i++) {//name.size()
            String url = "http://db.auto.sohu.com/shanghaivw/" + name.get(i)+"/";//得到车系页面
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .releaseTime(Long.valueOf(System.currentTimeMillis()))
                    .copyBizTags()
                    .needWashed(false)
                    .build();
            parsedLinks.add(record);
        }
    }

    private void carSeriesRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String carSeriesUrl = page.getRequest().getUrl();//车系url
        String koubeiUrl = "http:" + page.getHtml().xpath("//li[@id=\"model_dianping\"]/a/@href").get();//口碑url
        String brandName = page.getHtml().xpath("//span[@class=\"daohang\"]/a[3]/text()").get();//品牌名
        String brandUrl = page.getHtml().xpath("//span[@class=\"daohang\"]/a[3]/@href").get();//品牌url
        String subsidiaryBrandName = page.getHtml().xpath("//span[@class=\"daohang\"]/a[4]/text()").get();//子品牌名
        String carSeriesName = page.getHtml().xpath("//span[@class=\"daohang\"]/a[5]/text()").get();//车系名
        String carSeriesName2 = null;
        if (carSeriesName != null) {
            carSeriesName2 = subsidiaryBrandName + "-" + carSeriesName;//车系2
        }
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        HashMap<String, String> seriesMap = new HashMap<>();
        seriesMap.put(AutoVMCommonField.Field_Series_name, carSeriesName);//车系名
        seriesMap.put(AutoVMCommonField.Field_Series_url, carSeriesUrl);//车系url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series, seriesMap);//车系
        HashMap<String, String> brandMap = new HashMap<>();
        brandMap.put(AutoVMCommonField.Field_Brand_name, brandName);//品牌名
        brandMap.put(AutoVMCommonField.Field_Brand_url, brandUrl);//品牌url
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Brand, brandMap);//品牌
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Field_Praise_url, koubeiUrl);//口碑
        crawlerRecord.tagsCreator().bizTags().addCustomKV(AutoVMCommonField.Tag_Field_Series_Two, carSeriesName2);//二级车系
        crawlerRecord.tagsCreator().bizTags().addCustomKV("subsidiaryBrandName", subsidiaryBrandName);//子品牌


        CrawlerRequestRecord saleRecord = CrawlerRequestRecord.builder()//解析车型
                .itemPageRequest(crawlerRecord)
                .recordKey(carSeriesUrl)
                .httpUrl(carSeriesUrl)
                .releaseTime(System.currentTimeMillis())
                .resultLabelTag(article)
                .copyBizTags()
                .needWashed(true)
                .build();
        parsedLinks.add(saleRecord);

    }


    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
            crawlerDataList.addAll(saleWash(crawlerRecord, page));
        }
        if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
            crawlerDataList.addAll(stopWash(crawlerRecord, page));
        }

        return crawlerDataList;
    }

    //    解析在售款
    public List<CrawlerData> saleWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        String carSeriesUrl = page.getRequest().getUrl();//车系url
        List<Selectable> nodes = page.getHtml().xpath("//tr[@class=\"s_list\"]").nodes();
        for (Selectable node : nodes) {
            String saleUrl = "http:" + node.xpath(".//a[@uigs=\"cklb\"]/@href").get();
            String saleTitle = node.xpath(".//a[@uigs=\"cklb\"]/text()").get();
            String[] articleId = carSeriesUrl.split("/");
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", domain(), article, articleId[articleId.length - 1]))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(carSeriesUrl)
                    .addContentKV(AutoVMCommonField.Field_Model_name, saleTitle)
                    .addContentKV(AutoVMCommonField.Field_Model_url, saleUrl)
                    .addContentKV(AutoVMCommonField.Tag_Is_sale, "1")
                    .flowInPipelineTag("kafka")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        }
        return crawlerArticleDataList;
    }

    //    解析停售款
    public List<CrawlerData> stopWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        String carSeriesUrl = page.getRequest().getUrl();//车系url
        List<Selectable> nodes = page.getHtml().xpath("//td[@class=\"ftdleft\"]").nodes();
        for (Selectable node : nodes) {
            String saleUrl = "http:" + node.xpath(".//a[@uigs=\"cklb\"]/@href").get();
            String saleTitle = node.xpath(".//a[@uigs=\"cklb\"]/text()").get();
            String[] articleId = carSeriesUrl.split("/");
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", domain(), article, articleId[articleId.length - 1]))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(carSeriesUrl)
                    .addContentKV(AutoVMCommonField.Field_Model_name, saleTitle)
                    .addContentKV(AutoVMCommonField.Field_Model_url, saleUrl)
                    .addContentKV(AutoVMCommonField.Tag_Is_sale, "0")
                    .flowInPipelineTag("kafka")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        }

        return crawlerArticleDataList;
    }


    @Override
    public void afterExecute(CrawlerRecordContext context) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 50) {
                log.error("pcauto download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey() + count)
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }
}
